{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OOB 实践 - 搜索引擎"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.1 搜索引擎实践"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SearchEngineBase(object):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def add_corpus(self,file_path):\n",
    "        with open(file_path,'r') as fin:\n",
    "            text = fin.read()\n",
    "        self.process_corpus(file_path, text)\n",
    "    \n",
    "    def process_corpus(self,id,text):\n",
    "        raise Exception('process_corpus not implemented')\n",
    "    \n",
    "    def search(self, query):\n",
    "        raise Exception('search not implemented')\n",
    "    \n",
    "def main(search_engine):\n",
    "    for file_path in ['1.txt','2.txt','3.txt','4.txt','5.txt']:\n",
    "        search_engine.add_corpus('./doc/'+file_path)\n",
    "    \n",
    "    query = 'freedom'\n",
    "    results = search_engine.search(query)\n",
    "    print('found {} results(s):'.format(len(results)))\n",
    "    for result in results:\n",
    "        print(result)\n",
    "\n",
    "    # while True:\n",
    "    #     query = 'freedom'\n",
    "    #     results = search_engine.search(query)\n",
    "    #     print('found {} results(s):'.format(len(results)))\n",
    "    #     for result in results:\n",
    "    #         print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found 2 results(s):\n",
      "./doc/4.txt\n",
      "./doc/5.txt\n"
     ]
    }
   ],
   "source": [
    "class SimpleEngine(SearchEngineBase):\n",
    "    def __init__(self):\n",
    "        super(SimpleEngine,self).__init__()\n",
    "        self.__id_to_texts = {}\n",
    "    \n",
    "    def process_corpus(self,id, text):\n",
    "        self.__id_to_texts[id] = text\n",
    "    \n",
    "    def search(self,query):\n",
    "        results = []\n",
    "        for id,text in self.__id_to_texts.items():\n",
    "            if query in text:\n",
    "                results.append(id)\n",
    "        return results\n",
    "\n",
    "search_engine = SimpleEngine()\n",
    "main(search_engine)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found 2 results(s):\n",
      "./doc/4.txt\n",
      "./doc/5.txt\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "class BOWEngine(SearchEngineBase):\n",
    "    def __init__(self):\n",
    "        super(BOWEngine, self).__init__()\n",
    "        self.__id_to_words = {}\n",
    "\n",
    "    def process_corpus(self, id, text):\n",
    "        self.__id_to_words[id] = self.parse_text_to_words(text)\n",
    "\n",
    "    def search(self, query):\n",
    "        query_words = self.parse_text_to_words(query)\n",
    "        results = []\n",
    "        for id, words in self.__id_to_words.items():\n",
    "            if self.query_match(query_words, words):\n",
    "                results.append(id)\n",
    "        return results\n",
    "    \n",
    "    @staticmethod\n",
    "    def query_match(query_words, words):\n",
    "        for query_word in query_words:\n",
    "            if query_word not in words:\n",
    "                return False\n",
    "        return True\n",
    "\n",
    "    @staticmethod\n",
    "    def parse_text_to_words(text):\n",
    "        # 使用正则表达式去除标点符号和换行符\n",
    "        text = re.sub(r'[^\\w ]', ' ', text)\n",
    "        # 转为小写\n",
    "        text = text.lower()\n",
    "        # 生成所有单词的列表\n",
    "        word_list = text.split(' ')\n",
    "        # 去除空白单词\n",
    "        word_list = filter(None, word_list)\n",
    "        # 返回单词的 set\n",
    "        return set(word_list)\n",
    "\n",
    "search_engine = BOWEngine()\n",
    "main(search_engine)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'pylru'",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mModuleNotFoundError\u001b[39m                       Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[13]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpylru\u001b[39;00m\n\u001b[32m      3\u001b[39m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mLRUCache\u001b[39;00m(\u001b[38;5;28mobject\u001b[39m):\n\u001b[32m      4\u001b[39m     \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, size=\u001b[32m32\u001b[39m):\n",
      "\u001b[31mModuleNotFoundError\u001b[39m: No module named 'pylru'"
     ]
    }
   ],
   "source": [
    "import pylru\n",
    "\n",
    "class LRUCache(object):\n",
    "    def __init__(self, size=32):\n",
    "        self.cache = pylru.lrucache(size)\n",
    "    \n",
    "    def has(self, key):\n",
    "        return key in self.cache\n",
    "    \n",
    "    def get(self, key):\n",
    "        return self.cache[key]\n",
    "    \n",
    "    def set(self, key, value):\n",
    "        self.cache[key] = value\n",
    "\n",
    "class BOWInvertedIndexEngineWithCache(BOWInvertedIndexEngine, LRUCache):\n",
    "    def __init__(self):\n",
    "        super(BOWInvertedIndexEngineWithCache, self).__init__()\n",
    "        LRUCache.__init__(self)\n",
    "    \n",
    "    def search(self, query):\n",
    "        if self.has(query):\n",
    "            print('cache hit!')\n",
    "            return self.get(query)\n",
    "        \n",
    "        result = super(BOWInvertedIndexEngineWithCache, self).search(query)\n",
    "        self.set(query, result)\n",
    "        \n",
    "        return result\n",
    "\n",
    "search_engine = BOWInvertedIndexEngineWithCache()\n",
    "main(search_engine)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
